import scrapy
import re
from scrapy.selector import Selector
from CLC.items import ClcItem
from scrapy.spidermiddlewares.httperror import HttpError
from twisted.internet.error import (
    ConnectError,
    ConnectionDone,
    ConnectionLost,
    DNSLookupError,
    TCPTimedOutError,
)


class ClcindexSpider(scrapy.Spider):
    name = 'clcindex'
    allowed_domains = ['www.clcindex.com']
    start_urls = ['https://www.clcindex.com/']

    def start_requests(self):
        for url in self.start_urls:
            print(url)
            self.logger.info(url)
            request = scrapy.Request(url=url, callback=self.parse, cb_kwargs=dict(
                pcode=""), errback=self.parseError)
            request.cb_kwargs["level"] = 0
            yield request

    def parse(self, response, pcode, level):
        items = response.xpath('//*[@id="catTable"]/tbody/tr').getall()
        for x in items:
            code = Selector(text=x).xpath('normalize-space(//td[2]/text())').get()
            name = Selector(text=x).xpath('//td[3]/a/text()').get()
            url = Selector(text=x).xpath('//td[3]/a/@href').get()
            if url:
                request = scrapy.Request(url='https://www.clcindex.com' + url, callback=self.parse, cb_kwargs=dict(pcode=code), errback=self.parseError)
                request.cb_kwargs["level"] = level + 1
                yield request
                yield ClcItem(code=code, name=name, pcode=pcode, level=level)
            

            
    def parseError(self, failure):
        self.errorHandler(failure)

    def errorHandler(self, failure):
        # 日志记录所有的异常信息
        self.logger.error(repr(failure))

        # 假设我们需要对指定的异常类型做处理，
        # 我们需要判断异常的类型

        if failure.check(HttpError):
            # HttpError由HttpErrorMiddleware中间件抛出
            # 可以接收到非200 状态码的Response
            response = failure.value.response
            self.logger.error('HttpError on %s', response.url)

        elif failure.check(DNSLookupError):
            # 此异常由请求Request抛出
            request = failure.request
            self.logger.error('DNSLookupError on %s', request.url)

        elif failure.check(TimeoutError, TCPTimedOutError):
            request = failure.request
            self.logger.error('TimeoutError on %s', request.url)
